# -*- coding: utf-8 -*-
"""
Created on Thu May  9 18:03:51 2019

返回响应编码转换

@author: hhao
"""

import requests
from lxml import etree


class urlSpider(object):
    def __init__(self):
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
        self.base_url = "https://www.baidu.com/baidu?wd="
        self.key_wd = input("请输入要查找的网址类型的对应关键字:")

    def send_request(self, url):
        response = requests.get(url, headers=self.headers)
        return response

    # def parse_url(self, response):
    #     html = response.content
    #     html_obj = etree.HTML(html)
    #     url_list = html_obj.xpath("//a[@class='c-showurl']")
    #     return url_list

    def save_url(self, response):
        print("正在保存地址")
        html = response.content
        html_obj = etree.HTML(html)
        url_list = html_obj.xpath("//a[@class='c-showurl']/text()")
        print(url_list)

        #        for item in url_list:
        #            print(item)
        #            print(type(item))

        filename = open('111.txt', 'wb')
        for url in url_list:
            # filename.write(url.encode('utf8'))
            # """
            #     '\xa0' 中文空白符   的删除或转换
            # """
            # print("*"*50)
            # print(url.encode('utf8').decode('utf8').strip())
            # print(url.strip())
            # print("*"*50)
            # print(type(url))
            # filename.write(url.strip().encode('utf8'))

            url = url.strip().encode('utf8').decode('utf8') + "\n"
            filename.write(url.encode('utf8'))

        filename.close()
        # with open('111.txt', 'wb') as f:
        #     f.write(response.content)

    def main(self):
        full_url = self.base_url + self.key_wd
        response = self.send_request(full_url)
        self.save_url(response)


if __name__ == '__main__':
    web_url = urlSpider()
    web_url.main()
